#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :tcmap_com.py
# @Time      :2023/11/13 
# @Author    :CL
# @email     :1037654919@qq.com
# from https://blog.csdn.net/whatsuo/article/details/117331344
# 爬取 'http://www.tcmap.com.cn/'
import json

import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import random
from utils import mongo_manager
tcmap_com = mongo_manager("tcmap_com",db='public_data')
#
proxies = {'http':'http://127.0.0.1:15732',
          'https':'http://127.0.0.1:15732'}


def get_proxy():
    # API链接    后台获取链接地址
    proxyAPI = "http://cy123456.user.xiecaiyun.com/api/proxies?action=getJSON&key=NPD9167B5C&count=1&word=&rand=true&norepeat=false&detail=false&ltime=&idshow=false"
    proxyusernm = "cy123456"  # 代理帐号
    proxypasswd = "spider123"  # 代理密码
    # 获取IP
    r = requests.get(proxyAPI)
    if (r.status_code == 200):
        j = json.loads(r.text)
        if (j["success"] and len(j["result"]) > 0):
            p = j["result"][0]
            # name = input();
            proxyurl = "http://" + proxyusernm + ":" + proxypasswd + "@" + p["ip"] + ":" + "%d" % p["port"]
            return {'http': proxyurl, 'https': proxyurl}


ua_list = [
    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",#Chrome
    "Mozilla/5.0 (Windows NT 6.1; rv:40.0) Gecko/20100101 Firefox/40.0",#firwfox
    "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko",#IE
    "Opera/9.99 (Windows NT 5.1; U; zh-CN) Presto/9.9.9",#Opera
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0"
]
headers = {
    "User-Agent":random.choice(ua_list),
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
    "Accept-Encoding": "gzip, deflate",
    "Referer": "http://www.tcmap.com.cn/beijing/",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
    "Pragma": "no-cache",
    "Cache-Control": "no-cache"
}
cookies = {
    "__tins__21327377": "%7B%22sid%22%3A%201699854072060%2C%20%22vd%22%3A%203%2C%20%22expires%22%3A%201699856190259%7D",
    "__51cke__": "",
    "__51laig__": "11"
}
base_url = 'http://www.tcmap.com.cn/'
citys = []
distrcts = []
streets = []

def getStrongItem(url='http://www.tcmap.com.cn/zhejiangsheng'):
    print(f'begin {url}')
    html=requests.get(url, headers=headers,proxies=proxies,cookies=cookies)
    html.encoding = 'gb18030'
    soup = BeautifulSoup ( html.text, 'html.parser',from_encoding="gb18030" )
    href_limit = re.compile(".*<a class=\"blue\" href=\".*</a>.*")
    page=soup.find_all('table')
    items = []
    for p in page:
        lines = p.find_all('strong')
        for line in lines:
            if  (re.match(href_limit,str(line))) is not None:
                item_html = str(line).split('"')[3]
                item =  str(line).split('>')[-3].split('<')[0]
                items.append({'name':item,'href': 'http://www.tcmap.com.cn'+item_html})
    return items



def get_point_info(point={}):
    url = point['href']
    print(f'begin {url}')
    html=requests.get(url,proxies=proxies, headers=headers)
    html.encoding = 'gb18030'
    # print(html.text)
    if '对不起，你要查看的页面不存在！' in str(html.text):
        point['status'] = 'noPage'
        print(point['status'] )
        return point
    soup = BeautifulSoup ( html.text, 'html.parser', )
    tables=soup.find_all('table')
    # intruduce
    ps = soup.find_all('p')
    intru = ""
    for p in ps:
        intru += p.get_text().strip() + '\n'
    point['intru'] = intru.strip()
    # info = tables[1]
    tds = tables[1].find_all('td')
    infos = {}
    for td in tds:
        text = td.text.strip()
        try:
            infos[re.split(':|：',text)[0]] = re.split(':|：',text)[1]  # '：'
        except:
            pass
    point.update(infos)
    try:
        # citys = tables[2]
        tbl = pd.read_html(tables[2].prettify(), header=0)[0]
        citys_infos = [{j: str(row[j]) for j in tbl.columns} for i, row in tbl.iterrows()]
        point['contain_infos'] = citys_infos
    except Exception as e:
        # print(e)
        pass
    point['status'] = 'success'
    print(point['status'])
    return point




def getAll_point():
    # 五次循环 非常不健康
    for province in get_province():
        print(province)
        province['_id'] = province['href']
        province['level'] ='province'
        try:
            tcmap_com.insertOne(province)
        except Exception as e:
            print(e)
        citys= getStrongItem(province['href'])

        for city in citys:
            print(city)
            city["_id"] = city['href']
            city['level'] = 'city'
            try:
                tcmap_com.insertOne(city)
            except Exception as e:
                print(e)
            distrcts = getStrongItem(city['href'])
            for distrct in distrcts:
                distrct["_id"] = distrct['href']
                distrct['level'] = 'distrct'
                try:
                    tcmap_com.insertOne(distrct)
                except Exception as e:
                    print(e)
                streets = getStrongItem(distrct['href'])
                for street in streets:
                    street["_id"] = street['href']
                    street['level'] = 'street'
                    try:
                        tcmap_com.insertOne(street)
                    except Exception as e:
                        print(e)
                    communitys = getStrongItem(street['href'])
                    for community in communitys:
                        community["_id"] = community['href']
                        community['level'] = 'community'
                        try:
                            tcmap_com.insertOne(community)
                        except Exception as e:
                            print(e)


def get_province(url = 'http://www.tcmap.com.cn/'):
    """
    获取中国行政区划网省
    :return:province_list
    """
    province_list=[]
    response = requests.get(url, headers=headers, cookies=cookies)
    response.encoding = 'gb18030'
    soups = BeautifulSoup(response.text, 'lxml',from_encoding='gb18030')
    tables = soups.find_all('table')
    for table in tables:
        datas = table.find_all('div',class_='ht')
        for data in datas:
            try:
                name = data.find('a').get_text()
                herf = data.find('a').get('href')
                if '】' not in str(name):
                    province_list.append({'name':name,'href':'http://www.tcmap.com.cn'+herf})
            except:
                pass
    return province_list
def linshi():
    seeds = tcmap_com.findAll({'level':'street','xiangxi':None})
    for seed in seeds:
        # print(seed)
        province = seed['href'].replace('http://www.tcmap.com.cn/', '').split('/')[0]
        if province in ['beijing','tianjin','shanghai','chongqing']: # 直辖市跳过
            continue
        if province in ['hebei','shanxisheng','neimenggu','liaoning',
                        'jilin', 'heilongjiang','jiangsu', 'zhejiangsheng', 'anhui',
                        'fujian', 'jiangxi','shandong','henan','hubei'
                        ]: # 已经跑的跳过
            continue
        communitys = getStrongItem(seed['href'])
        for community in communitys:
            community["_id"] = community['href']
            community['level'] = 'community'
            try:
                tcmap_com.insertOne(community)
            except Exception as e:
                print(e)
        tcmap_com.updateOne({'_id':seed['_id']},{'xiangxi':1})
def main(): # 更新所有point 的信息
    while  True:
        points = tcmap_com.findAll({'status':None}).limit(10000)
        lens=0
        for point in points:

            lens += 1
            try:
                point = get_point_info(point)
                tcmap_com.updateOne({'_id':point['_id']},point)
            except Exception as e:
                print(e)
        if lens < 10000:
            break

def main_pool(): # 更新所有point 的信息 todo pool
    while  True:
        points = tcmap_com.findAll({'status':None}).limit(10)
        lens=0
        for point in points:
            lens += 1
            point = get_point_info(point)
            point['status'] = 'success'
            try:
                tcmap_com.updateOne({'_id':point['_id']},point)
            except Exception as e:
                print(e)
        if lens < 10000:
            break
if __name__ == '__main__':

    # print(province :=get_province())
    # getAll_point()
    # linshi() #
    main()
    tcmap_com.close()
    # print(getStrongItem())
